#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
from typing import Iterable

import parsel
import scrapy
from scrapy import Request

from apps.ding_tax.ding_tax.items import NetDingTaxArticleItem, NetDingTaxArticleUpdateItem, NetDingTaxRelationshipItem


class DingTaxSpider(scrapy.Spider):
    name = "ding_tax_spider"

    def start_requests(self) -> Iterable[Request]:
        url = "https://ai.wetax.com/bot/api/v1/policy/getArticleTabV2"
        yield scrapy.FormRequest(url=url, callback=self.parse_tags)

    def parse_tags(self, response, **kwargs):
        root = response.json().get("root")
        if root:
            tabs = root.get("tabs")
            for tab in tabs:
                value = tab["value"]
                name = tab["name"]
                url = "https://ai.wetax.com/bot/api/v1/policy/getArticleTags"
                data = {
                    "tabValues": [value],
                    "tags": [],
                    "keyword": "",
                    "searchMode": 10,
                    "issueNo": "",
                    "issueDateStart": "",
                    "issueDateEnd": "",
                    "isAll": True,
                    "pageSize": 10,
                    "pageNum": 1,
                }
                yield scrapy.FormRequest(
                    url=url,
                    body=json.dumps(data),
                    method="POST",
                    callback=self.parse_search_list,
                    cb_kwargs={
                        "tab_name_1": name,
                        "tab_value_1": value,
                    },
                )

    def parse_search_list(self, response, **kwargs):
        resp = response.json()
        root = resp.get("root")
        tab_name_1 = kwargs.get("tab_name_1")
        tab_value_1 = kwargs.get("tab_value_1")
        if root:
            for tabs in root:
                tab_name_2 = tabs.get("name")
                tab_value_2 = tabs.get("id")

                url = "https://ai.wetax.com/bot/api/v1/policy/search"
                data = {
                    "tabValues": [tab_value_1],
                    "tags": [tab_value_2],
                    "keyword": "",
                    "searchMode": 10,
                    "issueNo": "",
                    "issueDateStart": "",
                    "issueDateEnd": "",
                    "isAll": True,
                    "pageSize": 10,
                    "pageNum": 1,
                }
                yield scrapy.FormRequest(
                    url=url,
                    body=json.dumps(data),
                    method="POST",
                    callback=self.parse,
                    cb_kwargs={
                        "page": 1,
                        "tab_name_1": tab_name_1,
                        "tab_value_1": tab_value_1,
                        "tab_name_2": tab_name_2,
                        "tab_value_2": tab_value_2,
                    },
                )

    def parse(self, response, **kwargs):
        resp = response.json()
        tab_name_1 = kwargs.get("tab_name_1")
        tab_value_1 = kwargs.get("tab_value_1")
        tab_name_2 = kwargs.get("tab_name_2")
        tab_value_2 = kwargs.get("tab_value_2")
        yield from self.parse_detail_json(response, **kwargs)
        page = kwargs.get("page")
        total = (resp.get("root") or {}).get("total")
        if total > 10 * (page + 1):
            url = "https://ai.wetax.com/bot/api/v1/policy/search"
            data = {
                "tabValues": [tab_value_1],
                "tags": [tab_value_2],
                "keyword": "",
                "searchMode": 10,
                "issueNo": "",
                "issueDateStart": "",
                "issueDateEnd": "",
                "isAll": True,
                "pageSize": 10,
                "pageNum": page + 1,
            }
            yield scrapy.FormRequest(
                url=url,
                body=json.dumps(data),
                method="POST",
                callback=self.parse,
                cb_kwargs={
                    "page": page + 1,
                    "tab_name_1": tab_name_1,
                    "tab_value_1": tab_value_1,
                    "tab_name_2": tab_name_2,
                    "tab_value_2": tab_value_2,
                },
            )

    def parse_detail_json(self, response, **kwargs):
        tab_name_1 = kwargs.get("tab_name_1")
        tab_value_1 = kwargs.get("tab_value_1")
        tab_name_2 = kwargs.get("tab_name_2")
        tab_value_2 = kwargs.get("tab_value_2")
        articles = response.json()["root"]["articles"]
        for article in articles:
            encrypt_article_id = article["id"]
            # title = article["title"]
            # content = article["content"]
            # content_type = article["contentType"]
            # issue_org = article["issueOrg"]
            # issue_no = article["issueNo"]
            # issue_date = article["issueDate"]
            url = f"https://ai.wetax.com/bot/api/v1/home/checked_article_content?id={encrypt_article_id}"
            yield scrapy.Request(
                url=url,
                method="GET",
                callback=self.parse_detail,
                cb_kwargs={
                    "tab_name_1": tab_name_1,
                    "tab_value_1": tab_value_1,
                    "tab_name_2": tab_name_2,
                    "tab_value_2": tab_value_2,
                },
            )
            url = f"https://ai.wetax.com/bot/api/v1/policy/queryRelationship?encryptArticleId={encrypt_article_id}"
            yield scrapy.Request(
                url=url,
                method="GET",
                callback=self.parse_realtion_ship,
            )

    def parse_detail(self, response, **kwargs):
        """
            {
            "success": true,
            "root": {
                "areaId": "LJiy7jPyVu4YcWGy62Yk1Z",
                "articleType": "1",
                "title": "关于《国家税务总局关于推广境外旅客购物离境退税“即买即退”服务措施的公告》的解读",
                "remark": "",
                "status": 1.0,
                "valIdFlag": 0.0,
                "originalUrl": "https://fgk.chinatax.gov.cn/zcfgk/c100015/c5239601/content.html",
                "issueNo": "",
                "issueOrg": "货物和劳务税司",
                "issueDate": "2025-04-04 00:00:00",
                "content": "<p style=\"text-align: justify; text-indent: 2em;\">为进一步提升境外旅客购物离境退税便利水平，优化境外旅客消费体验，国家税务总局印发了《国家税务总局关于推广境外旅客购物离境退税“即买即退”服务措施的公告》（以下称《公告》）。现对有关内容解读如下：</p><p style=\"text-align: justify; text-indent: 2em;\"><strong>一、什么是离境退税？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">离境退税，是指境外旅客在离境口岸离境时，对其在退税商店购买的退税物品退还增值税。</p>\n<p style=\"text-align: justify; text-indent: 2em;\">境外旅客，是指在我国境内连续居住不超过183天的外国人和港澳台同胞。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>二、什么是离境退税“即买即退”？享受该服务的境外旅客需满足什么条件？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">离境退税“即买即退”服务措施（以下称离境退税“即买即退”）是离境退税的一项便利化举措，是指境外旅客在已实施离境退税政策地区的“即买即退”商店购买退税物品时，签订协议书并办理信用卡预授权后，即可在该“即买即退”商店现场申领与退税款等额的人民币款项（以下称预付金）。</p>\n<p style=\"text-align: justify; text-indent: 2em;\">享受该服务措施的境外旅客需满足以下条件：在离境时经海关验核通过、按照协议书约定在承诺期限内于指定口岸离境且符合离境退税的政策规定。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>三、能享受离境退税“即买即退”的物品需要满足哪些条件？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">适用现行离境退税政策的退税物品均属于离境退税“即买即退”范畴。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>四、成为“即买即退”商店需要满足哪些条件？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">已实施离境退税“即买即退”地区的退税商店，有意愿提供“即买即退”服务的，在与本地退税代理机构就预付金等事项达成一致后，即可成为“即买即退”商店。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>五、境外旅客办理离境退税“即买即退”的流程</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">享受离境退税“即买即退”的境外旅客，办理该业务时，主要涉及三个环节：<span style=\"font-weight: bold;\">一是商店开单预付。</span>申请离境退税“即买即退”的境外旅客签订协议书后，由商店为境外旅客办理信用卡预授权手续，开具《境外旅客购物离境退税申请单》，并现场支付预付金。<span style=\"font-weight: bold;\">二是海关进行验核。</span>享受离境退税“即买即退”的境外旅客，在离境时应当持退税物品、《境外旅客购物离境退税申请单》、退税物品销售发票和本人有效身份证件，向海关申报并接受海关验核。<span style=\"font-weight: bold;\">三是退税代理机构审核。</span>境外旅客应将本人护照等有效身份证件、海关验核签章的《境外旅客购物离境退税申请单》、退税物品销售发票，提交给离境口岸隔离区内的退税代理机构。退税代理机构对相关材料信息进行审核，对旅客按照协议书约定在承诺期限内于指定口岸离境，且符合现行离境退税政策规定的，退税代理机构为其解除信用卡预授权，办结退税事项，预付金视为已退税款。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>六、什么是协议书？协议书必须是由境外旅客本人签订吗？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">协议书是由选择享受离境退税“即买即退”的境外旅客签订的，声明本人知悉离境退税“即买即退”权责事项，承诺在约定期限内自指定口岸离境，并同意以信用卡预授权为担保，按照约定限额和支付方式取得预付金的文书。协议书由“即买即退”商店向境外旅客免费提供。</p>\n<p style=\"text-align: justify; text-indent: 2em;\">协议书必须由境外旅客本人签订。“即买即退”商店或退税代理机构应妥善留存境外旅客签订的协议书。协议书的留存单位由“即买即退”商店与退税代理机构协商确定。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>七、什么是信用卡预授权？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">信用卡预授权，是指商店或代理机构将预付金支付给享受离境退税“即买即退”的境外旅客，并在其信用卡冻结相应额度。</p>\n<p style=\"text-align: justify; text-indent: 2em;\">信用卡预授权由“即买即退”商店或退税代理机构为境外旅客办理。预授权金额应与其申领的预付金等额。境外旅客信用卡预授权的解除、扣款，由“即买即退”商店所在地提供离境退税“即买即退”的退税代理机构负责。“即买即退”商店及退税代理机构应妥善留存信用卡预授权业务相关的银行单据。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>八、什么是预付金？境外旅客取得预付金的方式有哪些？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">预付金，是在“即买即退”商店现场向享受离境退税“即买即退”的境外旅客支付的，与退税款等额的人民币款项。具体形式包括现金支付和电子支付。</p>\n<p style=\"text-align: justify; text-indent: 2em;\">预付金由“即买即退”商店或者退税代理机构先行垫付，垫付方、具体形式及限额标准由各省级税务局、“即买即退”商店与退税代理机构协商确定。</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>九、境外旅客取得预付金的金额应如何计算？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">境外旅客可取得的预付金金额，是以退税物品的增值税普通发票金额(含增值税)为依据计算的实退增值税额，即：</p>\n<p style=\"text-align: justify; text-indent: 2em;\">预付金金额=实退增值税额</p>\n<p style=\"text-align: justify; text-indent: 2em;\">实退增值税额=退税物品销售发票金额（含增值税）×退税率－退税代理机构办理退税手续费</p>\n<p style=\"text-align: justify; text-indent: 2em;\"><strong>十、境外旅客未履行协议书约定，超过承诺期限或未自指定口岸离境的，应该如何处理？</strong></p>\n<p style=\"text-align: justify; text-indent: 2em;\">对旅客未履行协议书约定，超过承诺期限或未自指定口岸离境的，退税代理机构自旅客承诺离境期限期满之日起3个工作日内，通过信用卡预授权扣款方式追回预付金。待旅客实际离境时，再由离境地退税代理机构按照现行规定审核办理退税。</p>",
                "contentType": 10.0,
                "creatorId": "XZDL7Rop6jQaK9BkpwT7HV",
                "creator": "botLogin15606757856",
                "creationDate": "2025-04-09 09:18:11",
                "modifierId": "XZDL7Rop6jQaK9BkpwT7HV",
                "modifier": "木繁",
                "modificationDate": "2025-04-09 09:21:28",
                "syncFlag": 0.0,
                "id": "C4isqy73qUzQdHsprfkeFA"
            }
        }
        """
        tab_name_1 = kwargs.get("tab_name_1")
        tab_name_2 = kwargs.get("tab_name_2")
        resp = response.json()
        root = resp.get("root")
        effective_status_map = {
            0: "全文有效",
            1: "部分有效",
            2: "全文失效",
            3: "已被修订",
        }
        if root:
            content_type = f"{tab_name_1}" if tab_name_2 else f"{tab_name_1}_{tab_name_2}"
            effective_status = effective_status_map.get(root.get("valIdFlag"))
            text = root.get("content")
            article_item = {
                "area_id": root.get("areaId"),
                "article_type": root.get("articleType"),
                "title": root.get("title"),
                "status": root.get("status"),
                "effective_status": effective_status,
                "link": root.get("originalUrl"),
                "document_number": root.get("issueNo"),
                "issuing_unit": "|".join(root.get("issueOrg").split(" ")) if root.get("issueOrg") else "",
                "release_time": root.get("issueDate"),
                "text": text,
                "content_type": content_type,
                "encrypt_article_id": root.get("id"),
                "remark": root.get("remark"),
                # "article_id": root.get("id"),
                # "attachment_link": root.get("id"),
            }
            yield NetDingTaxArticleItem(**article_item)
            a_list = parsel.Selector(text).xpath("//a[@attachment_id]")
            for a in a_list:
                attachment_encrypt_id = response.urljoin(a.attrib.get("attachment_encryptId"))
                attachment_link = f"https://ai.wetax.com/bot/api/v1/policy/searchFileDetail?fileId={attachment_encrypt_id}&fromKbsFile=false"
                yield scrapy.Request(
                    url=attachment_link,
                    callback=self.parse_article_detail_attachment_link,
                    meta={"article_item": article_item},
                )

    def parse_article_detail_attachment_link(self, response, **kwargs):
        resp = response.json()
        root = resp.get("root")
        if root:
            article_item = kwargs.get("article_item")
            article_item.update(
                {
                    "attachment_link": root.get("url"),
                }
            )
            yield NetDingTaxArticleItem(**article_item)

    def parse_realtion_ship(self, response, **kwargs):
        """
        {
            "success": true,
            "root": {
                "allRelationshipsList": [
                    {
                        "startNodeId": 7112,
                        "baseContactId": 10006,
                        "articleContactId": 144795,
                        "baseContactName": "相关",
                        "colourType": 10,
                        "endNodeId": 7113
                    }
                ],
                "allNodesList": [
                    {
                        "isCenter": false,
                        "encryptArticleId": "RJEtPdyEbfY8XsRoPnrSrG",
                        "id": 7113,
                        "title": "国家税务总局关于推广境外旅客购物离境退税“即买即退”服务措施的公告"
                    },
                    {
                        "isCenter": true,
                        "encryptArticleId": "C4isqy73qUzQdHsprfkeFA",
                        "id": 7112,
                        "title": "关于《国家税务总局关于推广境外旅客购物离境退税“即买即退”服务措施的公告》的解读"
                    }
                ]
            }
        }
        """
        resp = response.json()
        root = resp.get("root")
        if root:
            nodes = root.get("allNodesList")
            relationships = root.get("allRelationshipsList")
            for node in nodes:
                node_item = {
                    "encrypt_article_id": node.get("encryptArticleId"),
                    "title": node.get("title"),
                    "article_id": node.get("id"),
                }
                yield NetDingTaxArticleUpdateItem(**node_item)

            for relationship in relationships:
                relation_ship_item = {
                    "start_node_id": relationship.get("startNodeId"),
                    "base_contact_id": relationship.get("baseContactId"),
                    "article_contact_id": relationship.get("articleContactId"),
                    "base_contact_name": relationship.get("baseContactName"),
                    "colour_type": relationship.get("colourType"),
                    "end_node_id": relationship.get("endNodeId"),
                }
                yield NetDingTaxRelationshipItem(**relation_ship_item)


if __name__ == "__main__":
    from scrapy import cmdline

    cmdline.execute(["scrapy", "crawl", "ding_tax_spider"])
